package com.ybox.crawler.service;

import com.ybox.common.core.exception.ServiceException;
import com.ybox.common.core.utils.StringUtils;
import com.ybox.common.core.utils.UrlSpiltUtils;
import com.ybox.common.core.utils.uuid.IdUtils;
import com.ybox.crawler.domain.Article;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;


/**
 * @Author ych
 * @create 2023-2023/4/2-11:22
 */
public class JuejinCrawlerAll implements PageProcessor {

    // 样式成功案例 https://juejin.cn/post/7216584694791454777?
    // https://juejin.cn/post/7217399337989488695
    // https://juejin.cn/post/7025085812517634062
    private static String URL = "https://juejin.cn/post/7025085812517634062";

    /**
     * 解析页面
     *
     * @param page
     */
    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        Document document = html.getDocument();

        String startHtml = "<html><head>";
        String endHtml = "</div></body></html>";
        // 将掘金的css样式提前保存在服务器当中
        String css = "<meta charset=\"utf-8\" />" +
                "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1, user-scalable=no, viewport-fit=cover\" />" +
                "<link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/app.5d6de2a.css\" />" +
                "<link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/default.2eed0bc.css\" />" +
                " <link rel=\"stylesheet\" href=\"http://127.0.0.1:9000/ybox-0/css/60.daee106.css\" />";

        String endHead = "</head><body style=\"padding-left: 2rem;padding-right: 2rem;background-color: #fff;\">" +
                "<div class=\"main-area article-area\" data-v-2f7a31cd=\"\" data-v-5811513e=\"\">";


        // 保存原网页
        // byte bytes[] = new byte[512];
        // bytes = html.toString().getBytes();
        // save2File("D:/java/doc.html",bytes);

        Elements meta = document.select("meta");
        if (!StringUtils.contains(document.title(),"掘金")) {
            throw new ServiceException();
        }

        // 因为掘金HTML格式比较固定，所以可以写死
        Elements elements = document.select("img");
        HashMap<String, String> map = new HashMap<>(32);
        String path = null;
        for (Element element : elements) {
            String a = element.attr("src");
            if (a != null && !"".equals(a)) {
                if (map.get(a) == null || "".equals(map.get(a))) {
                    // 如果是base64图片
                    if (a.startsWith("data:image/")){
                        map.put(a, a);
                        continue;
                    }
                    try {
                        // 下载图片
                        if (UrlSpiltUtils.getUrlMap(a).isEmpty()){
                            String[] split = a.split("/");
                            int length = split.length;
                            String s = split[length - 1].split("\\.")[1];
                            s = s.replace("?","");
                            path = download(a, s);
                        }else {
                            path = download(a, UrlSpiltUtils.getUrlMap(a).get("wx_fmt"));
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                    map.put(a, path);
                }
            }
        }
        Elements content = document.getElementsByTag("article");
        Elements articleEnd = document.getElementsByClass("article-end");
        String contentHtml = content.toString();
        for (Map.Entry<String, String> entry : map.entrySet()) {
            contentHtml = contentHtml.replace(entry.getKey(), entry.getValue());
        }
        // 移除掘金的文章收录
        Elements columnContainer = articleEnd.select("div[class=column-container]");
        columnContainer.remove();
        // 移除掘金的拓展
        Elements extensionBanner = articleEnd.select("div[class=extension-banner]");
        extensionBanner.remove();
        String resultHtml = startHtml + css + endHead  + contentHtml + articleEnd + endHtml;
        String authors = content.select("span[class=name]").text();
        String titles = content.select("h1[class=article-title]").text();
        String img = content.select("img").attr("src");
        String description = content.select("time[class=time]").text();
        String source = URL;
        Article article = new Article(titles, authors, URL, resultHtml, img, source, description);
        // 保存处理后的数据
        page.putField("article", article);

        System.out.println("爬取文章完成！！");
    }

    private Site site = Site.me()
            // 设置编码
            .setCharset("utf8")
            // 设置超时时间
            .setTimeOut(10 * 1000)
            // 设置重试时间
            .setRetrySleepTime(10000)
            // 设置重试次数
            .setRetryTimes(5);

    @Override
    public Site getSite() {
        return site;
    }


    /**
     * 获取html图片
     *
     * @param urlString
     * @param suffix
     * @return
     * @throws Exception
     */
    public static String download(String urlString, String suffix) throws Exception {
        if (urlString.startsWith("//")){
            urlString = urlString.replace("//","https://");
        }
        // 获取URL并构造URL
        URL url = new URL(urlString);
        // 打开URL连接
        URLConnection con = url.openConnection();
        // 定义输入流
        InputStream is = con.getInputStream();
        // 定义1K的数据缓冲
        byte[] bs = new byte[1024];
        // 读取到的数据长度
        int len;
        /**
         *
         * 设置输出的文件流并设置下载路径及下载图片名称
         */
        String id = IdUtils.fastUUID();
        String filename = "D:\\java\\img\\" + id + "." + suffix;
        File file = new File(filename);
        FileOutputStream os = new FileOutputStream(file, true);
        // 开始读取
        while ((len = is.read(bs)) != -1) {
            os.write(bs, 0, len);
        }
        // 下载完毕，关闭所有链接
        os.close();
        is.close();
        String finalName = "./img/" + id + "." + suffix;
        return finalName;
    }


    /**
     * 主函数执行爬虫
     *
     * @param args
     */
    public static void main(String[] args) {
        Spider.create(new JuejinCrawlerAll())
                // 设置爬取的数据页面
                .addUrl(URL)
                .addPipeline(new Pipeline() {
                    /**
                     * 保存到文件
                     * @param path  文件名称
                     * @param bytes 字节流
                     * @throws IOException
                     */
                    private String path = "D:/java/juejin-test.html";

                    @Override
                    public void process(ResultItems resultItems, Task task) {
                        Article article = resultItems.get("article");

                        // 判断获取到的数据不为空
                        if (article != null) {
                            try {
                                // 如果有值则进行保存
                                File txt = new File(path);
                                if (!txt.exists()) {
                                    txt.createNewFile();
                                }
                                FileOutputStream fileOutputStream = new FileOutputStream(txt);
                                byte bytes[] = new byte[512];
                                // 格式化html
                                Document doc = Jsoup.parse(article.getResultHtml());
                                doc.outputSettings().prettyPrint(true);
                                String formattedHtml = doc.html();

                                bytes = formattedHtml.getBytes();
                                fileOutputStream.write(bytes);
                                fileOutputStream.close();
                            } catch (FileNotFoundException e) {
                                throw new RuntimeException(e);
                            } catch (IOException e) {
                                throw new RuntimeException(e);
                            }

                        }
                    }
                })
                .run();
    }
}
